/*
 * Copyright 2011-2012 by The Regents of the University of California Licensed
 * under the Apache License, Version 2.0 (the "License"); you may not use this
 * file except in compliance with the License. you may obtain a copy of the
 * License from
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package edu.ucsd.db.bassarisk.indexer.simple_wp;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.logging.Logger;

import edu.ucsd.db.bassarisk.indexer.api.ICorpus;
import edu.ucsd.db.bassarisk.indexer.api.ITextArtifact;

/**
 * Represents a collection of wikipedia articles in a simplified variant of the
 * wex extraction with XML tags removed, whitespace normalized, and diacritical
 * characters removed. See tool wex_xml_stripper.py for details on the
 * simplification.
 * 
 * @author nbales
 */
public class SimpleWpCorpus implements ICorpus {
  private BufferedReader inputReader = null;

  private final Logger LOG = Logger.getLogger(this.getClass().getName());

  @Override
  public void initialize(final InputStream in) {
    // The simplified-wex code is not necessarily supposed to be the fastest
    // or most memory efficient code, so we prefer a nice interface.
    this.inputReader = new BufferedReader(new InputStreamReader(in));
  }

  @Override
  public Iterator<ITextArtifact> iterator() {
    return new Iterator<ITextArtifact>() {
      protected String next = null;

      @Override
      public boolean hasNext() {
        if (this.next == null) {
          try {
            this.next = SimpleWpCorpus.this.inputReader.readLine();
          } catch (IOException e) {
            throw new RuntimeException(e);
          }
        }
        return this.next != null;
      }

      @Override
      public SimpleWpArticle next() {
        if (this.next == null) {
          if (!this.hasNext()) { throw new NoSuchElementException(); }
        }
        // A wikipedia article is stored in the simplified wex dump as two
        // values separated by a tab character: the wikipedia article id and
        // the encoded article content (including title).
        int tab = this.next.indexOf('\t');
        if (tab < 0) {
          // Invalid article format
          // Recover by skipping all subsequent poorly formatted articles
          int invalidArticleCount = 0;
          while (tab < 0 && this.hasNext()) {
            tab = this.next.indexOf('\t');
            invalidArticleCount++;
          }
          SimpleWpCorpus.this.LOG.warning(String
              .format("Encounterd {0} wex-simplified "
                  + "article(s) with invalid format.", invalidArticleCount));
          // We may have reached the end of the input
          if (this.next == null) {
            if (!this.hasNext()) { throw new NoSuchElementException(); }
          }
        }
        String wikpediaId = this.next.substring(0, tab);
        String articleContent = this.next.substring(tab + 1);
        this.next = null;
        return new SimpleWpArticle(wikpediaId, articleContent);
      }

      @Override
      public void remove() {
        // Collection is read only.
        String message =
            "Attempt to remove wikipedia article from read only "
                + "article collection ignored.";
        SimpleWpCorpus.this.LOG.warning(message);
        throw new UnsupportedOperationException(message);
      }
    };
  }

}
